{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle as pkl\n",
    "import joblib\n",
    "import numpy as np\n",
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0,1,2,3,4\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_feat_file = open('./Data/Processed/pos_feat.pkl','rb')\n",
    "pos_custom_id = pkl.load(pos_feat_file)\n",
    "pos_FN = pkl.load(pos_feat_file)\n",
    "pos_Active = pkl.load(pos_feat_file)\n",
    "pos_club_member_status = pkl.load(pos_feat_file)\n",
    "pos_fashion_news_frequency = pkl.load(pos_feat_file)\n",
    "pos_age = pkl.load(pos_feat_file)\n",
    "pos_artid = pkl.load(pos_feat_file)\n",
    "pos_pcode = pkl.load(pos_feat_file)\n",
    "pos_sales_channel_id = pkl.load(pos_feat_file)\n",
    "pos_isval = pkl.load(pos_feat_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "testrange ok\n",
      "valrange ok\n"
     ]
    }
   ],
   "source": [
    "\n",
    "testrange = [i for i in range(len(pos_isval)) if pos_isval[i] == 0]\n",
    "testrange = np.array(testrange)\n",
    "print('testrange ok')\n",
    "valrange = [i for i in range(len(pos_isval)) if pos_isval[i] == 1]\n",
    "valrange = np.array(valrange)\n",
    "print('valrange ok')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_pcode_seq_file = open('./Data/Processed/pos_pcode_seq.db','rb')\n",
    "pos_pcode_seq = joblib.load(pos_pcode_seq_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "shape = [(i+1)*1000000 for i in range(30)]\n",
    "shape.append(len(testrange))\n",
    "divide_testrange = np.split(testrange,shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(31):\n",
    "    pos_pcode_seq_divide = np.array([pos_pcode_seq[j] for j in divide_testrange[i]])\n",
    "    f=open(f'./Data/Processed/divide/pos_pcode_seq{i}.db','wb')\n",
    "    joblib.dump(pos_pcode_seq_divide,f)\n",
    "    f.close()\n",
    "    pos_pcode_seq_divide = None\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_pcode_seq_val = np.array([pos_pcode_seq[i] for i in valrange])\n",
    "f = open(f'./Data/Processed/divide/pos_pcode_seq_val.db','wb')\n",
    "joblib.dump(pos_pcode_seq_val,f)\n",
    "f.close()\n",
    "pos_pcode_seq_val = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(31):\n",
    "    pos_custom_id_divide = np.array([pos_custom_id[j] for j in divide_testrange[i]])\n",
    "    pos_FN_divide = np.array([pos_FN[j] for j in divide_testrange[i]])\n",
    "    pos_Active_divide = np.array([pos_Active[j] for j in divide_testrange[i]])\n",
    "    pos_club_member_status_divide = np.array([pos_club_member_status[j] for j in divide_testrange[i]])\n",
    "    pos_fashion_news_frequency_divide = np.array([pos_fashion_news_frequency[j] for j in divide_testrange[i]])\n",
    "    pos_age_divide = np.array([pos_age[j] for j in divide_testrange[i]])\n",
    "    pos_artid_divide = np.array([pos_artid[j] for j in divide_testrange[i]])\n",
    "    pos_pcode_divide = np.array([pos_pcode[j] for j in divide_testrange[i]])\n",
    "    pos_sales_channel_id_divide = np.array([pos_sales_channel_id[j] for j in divide_testrange[i]])\n",
    "    pos_feat_divide_file = open(f'./Data/Processed/divide/pos_feat{i}.db','wb')\n",
    "    joblib.dump(pos_custom_id_divide,pos_feat_divide_file)\n",
    "    joblib.dump(pos_FN_divide,pos_feat_divide_file)\n",
    "    joblib.dump(pos_Active_divide,pos_feat_divide_file)\n",
    "    joblib.dump(pos_club_member_status_divide,pos_feat_divide_file)\n",
    "    joblib.dump(pos_fashion_news_frequency_divide,pos_feat_divide_file)\n",
    "    joblib.dump(pos_age_divide,pos_feat_divide_file)\n",
    "    joblib.dump(pos_artid_divide,pos_feat_divide_file)\n",
    "    joblib.dump(pos_pcode_divide,pos_feat_divide_file)\n",
    "    joblib.dump(pos_sales_channel_id_divide,pos_feat_divide_file)\n",
    "    pos_custom_id_divide=None\n",
    "    pos_FN_divide=None\n",
    "    pos_Active_divide = None\n",
    "    pos_club_member_status_divide=None\n",
    "    pos_fashion_news_frequency_divide=None\n",
    "    pos_age_divide=None\n",
    "    pos_artid_divide=None\n",
    "    pos_pcode_divide=None\n",
    "    pos_sales_channel_id_divide=None\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "pos_custom_id_divide=np.array([pos_custom_id[i] for i in valrange])\n",
    "pos_FN_divide=np.array([pos_FN[i] for i in valrange])\n",
    "pos_Active_divide = np.array([pos_Active[i] for i in valrange])\n",
    "pos_club_member_status_divide = np.array([pos_club_member_status[i] for i in valrange])\n",
    "pos_fashion_news_frequency_divide = np.array([pos_fashion_news_frequency[i] for i in valrange])\n",
    "pos_age_divide = np.array([pos_age[i] for i in valrange])\n",
    "pos_artid_divide = np.array([pos_artid[i] for i in valrange])\n",
    "pos_pcode_divide = np.array([pos_pcode[i] for i in valrange])\n",
    "pos_sales_channel_id_divide = np.array([pos_sales_channel_id[i] for i in valrange])\n",
    "pos_feat_divide_file = open(f'./Data/Processed/divide/pos_feat_val.db','wb')\n",
    "joblib.dump(pos_custom_id_divide,pos_feat_divide_file)\n",
    "joblib.dump(pos_FN_divide,pos_feat_divide_file)\n",
    "joblib.dump(pos_Active_divide,pos_feat_divide_file)\n",
    "joblib.dump(pos_club_member_status_divide,pos_feat_divide_file)\n",
    "joblib.dump(pos_fashion_news_frequency_divide,pos_feat_divide_file)\n",
    "joblib.dump(pos_age_divide,pos_feat_divide_file)\n",
    "joblib.dump(pos_artid_divide,pos_feat_divide_file)\n",
    "joblib.dump(pos_pcode_divide,pos_feat_divide_file)\n",
    "joblib.dump(pos_sales_channel_id_divide,pos_feat_divide_file)\n",
    "pos_feat_divide_file.close()\n",
    "pos_custom_id_divide=None\n",
    "pos_FN_divide=None\n",
    "pos_Active_divide=None\n",
    "pos_club_member_status_divide=None\n",
    "pos_fashion_news_frequency_divide=None\n",
    "pos_age_divide=None\n",
    "pos_artid_divide=None\n",
    "pos_pcode_divide=None\n",
    "pos_sales_channel_id_divide=None\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_club_member_status = None\n",
    "pos_custom_id=None\n",
    "pos_fashion_news_frequency=None\n",
    "pos_FN=None\n",
    "pos_isval = None\n",
    "pos_pcode = None\n",
    "pos_pcode_seq =None\n",
    "pos_sales_channel_id=None\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "MemoryError",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_38312/554570387.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mpos_artid_seq_file\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'./Data/Processed/pos_artid_seq.pkl'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mpos_artid_seq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpkl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpos_artid_seq_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mMemoryError\u001b[0m: "
     ]
    }
   ],
   "source": [
    "pos_artid_seq_file = open('./Data/Processed/pos_artid_seq.pkl','rb')\n",
    "pos_artid_seq = joblib.load(pos_artid_seq_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(31):\n",
    "    pos_artid_seq_divide = np.array([pos_artid_seq[j] for j in divide_testrange[i]])\n",
    "    f=open(f'./Data/Processed/divide/pos_artid_seq{i}.db','wb')\n",
    "    joblib.dump(pos_artid_seq_divide,f)\n",
    "    f.close()\n",
    "    pos_artid_seq_divide = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_artid_seq_val = np.array([pos_artid_seq[i] for i in valrange])\n",
    "f = open(f'./Data/Processed/divide/pos_artid_seq_val.db','wb')\n",
    "joblib.dump(pos_artid_seq_val,f)\n",
    "f.close()\n",
    "pos_artid_seq_val = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_pcode_seq_divide_file = None\n",
    "pos_pcode_seq_val_file = None\n",
    "pos_feat_divide_file = None\n",
    "pos_feat_val_file = None"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "7c2e65e64076883662e9fbb467097aa6d81839e6b77012bdd0b6d8c4fbfb9623"
  },
  "kernelspec": {
   "display_name": "Python 3.7.11 ('SR-GNN')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
